#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# 爬取豆瓣书籍


from lxml import etree
import requests
import json
import time
import random
import re

# run flag
RUN = True

HOST = 'https://book.douban.com'
# 存储要执行的图书id(豆瓣网ID)
ids = []
# 存储已经完成信息采集的图书ID
done_ids = []

# 本地记录目标图书ID的文件
ids_path = 'ids.txt'
# 本地记录已完成信息收集的图书ID
done_path = 'done.txt'
# 本地记录收集失败的图书ID
fail_path = 'fail.txt'

start_time = None

def getBookInfo(id):
    """
        输入图书ID,从豆瓣网爬取图书出版信息
    """
    # get subject id from url
    url = 'https://book.douban.com/subject/' + id
    # requests html
    res = requests.get(url = url,timeout= 10)
    
    if res.text.find('异常请求从你的 IP 发出') > -1:
        print('%s : The website rejects the request and the program exits.' % get_time())
        global RUN
        RUN = False

    html = etree.HTML(res.text)
    info = {}
    # get book base info
    result = html.xpath('//*[@id="info"]//text()')
    infos = []
    for r in result:
        r = r.replace('\n','')
        r = r.replace(' ','')
        r = r.replace('\xa0','')
        if r != '':
            infos.append(r)

    info_str = '{'
    while len(infos) != 0:
        key = infos.pop(0)
        if key.find(':') >= 0:
            key = key.replace(':','')
            info_str += '"' + key + '":'
            key = infos.pop(0)
            key = key.replace(':','')
            info_str += '"' + key + '",'
        else:
            key = key.replace(':','')
            info_str += '"' + key + '"'
            key = infos.pop(0)
            key = key.replace(':','')
            info_str += ':'
            key = infos.pop(0)
            key = key.replace(':','')
            info_str += '"' + key + '",'
    info_str = info_str[:-1]
    info_str += '}'
    info_str = info_str.replace('作者','author')
    info_str = info_str.replace('出版社','publisher',1)
    info_str = info_str.replace('出品方','author')
    info_str = info_str.replace('原作名','book_name')
    info_str = info_str.replace('译者','translator')
    info_str = info_str.replace('出版年','pubdate')
    info_str = info_str.replace('页数','pages')
    info_str = info_str.replace('定价','price')
    info_str = info_str.replace('装帧','binding')
    info_str = info_str.replace('丛书','series')
    info_str = info_str.replace('ISBN','isbn')
    info = json.loads(info_str)

    # 转换数据类型
    try:
        info['price'] = re.findall('\d+\.?\d*', info['price'])
        info['price'] = float(info['price'][0])
    except BaseException as e:
        print(e)
        info['price'] = None
    try:
        info['pages'] = int(info['pages'])
    except:
        info['pages'] = None

    # get book name
    info['name'] = html.xpath('//*[@id="wrapper"]/h1/span')[0].text

    # get img
    image = html.xpath('//*[@id="mainpic"]/a/img/@src')[0]
    info['image'] = image

    # get ohter info

    result = html.xpath('//div[@class="intro"]')
    checkShort = html.xpath('//div[@id="link-report"]/*[@class="short"]')
    if len(checkShort) > 0:
        result.pop(0)

    # get book intro
    if len(result) > 0:
        el = result[0]
        intro = el.xpath('.//text()')
        intro = ''.join(intro)
        intro = intro.replace('\n','')
        intro = intro.replace(' ','')
        info['summary'] = intro

    # get author intro
    if len(result) > 1:
        el = result[1]
        author_info = el.xpath('.//text()')
        author_info = ''.join(author_info)
        author_info = author_info.replace('\n','')
        author_info = author_info.replace(' ','')
        info['authorIntro'] = author_info

    # get book catalog
    catalog = html.xpath('//*[@id="dir_' + id + '_full"]//text()')
    if len(catalog) == 0:
        catalog = html.xpath('//*[@id="dir_' + id + '_short"]//text()')
    catalog = ''.join(catalog)
    info['catalog'] = catalog

    return info

def saveBookInfo(info):
    """
        将图书出版信息保存到数据库中
    """
    url = 'http://wz818.cn:8080/bookisland/api/books'
    res = requests.post(url=url, data=info)
    return res.json()

def init():
    """
        读取本地记录
    """
    global done_ids
    global ids
    global start_time
    start_time = time.time()
    
    print('%s : try to read ' % get_time() + done_path )
    try:
        with open(done_path, 'r', encoding='utf-8') as f:
            line = f.readline()
            while line:
                done_ids.append(line.replace('\n',''))
                line = f.readline()
    except:
        print('%s : file ' % get_time() + done_path + ' is not exits. continue')

    print('%s : try to read ' % get_time() + ids_path )
    try:
        with open(ids_path, 'r', encoding='utf-8') as f:
            line = f.readline()
            while line:
                id = line.replace('\n','')
                if id not in ids:
                    ids.append(id)
                line = f.readline()
    except:
        print('%s : read file ' + ids_path + ' is not exits., program will be exit.' % get_time())
        exit(-1)

    


def get_time():
    return str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))

def get_show_time(tt):
    s = ''
    if int(tt/86400) > 0:
        s += str(int(tt/86400)) + '天'
        tt = tt%86400
    if int(tt/3600) > 0:
        s += str(int(tt/3600)) + '时'
        tt = tt%3600
    if int(tt/60) > 0:
        s += str(int(tt/60)) + '分'
        tt = tt%60
    s += str(int(tt)) + '秒'

    return s

def show_info(id, flag):
    len_residue = len(ids)
    len_done = len(done_ids)
    len_all = len_residue + len_done
    run_time = time.time() - start_time
    run_time = get_show_time(run_time)
    residue_time = get_show_time(len_residue * 8)
    if flag:
        flag = 'OK'
    else:
        flag = 'Fail'
    print('%s : %s is %s [%d/%d](Run time: %s , Remaining time : %s)' % \
        (get_time(), id, flag, len_done, len_all, run_time, residue_time))

def error_id(id, msg):
    with open(fail_path, 'a+', encoding='utf-8') as f:
        f.write(id + ':' + str(msg) + '\n')

def success_id(id):
    with open(done_path, 'a+', encoding='utf-8') as f:
        f.write(id + '\n')

if __name__ == '__main__':
    print('%s : Program start...' % get_time())
    
    # Execute the initializer
    init()
    print('%s : 目标数量: %d, 预计完成用时: %s' % (get_time(), len(ids), (get_show_time(len(ids) * 8))))
    while len(ids) > 0 and RUN:
        id = ids.pop(0)
        if id in done_ids:
            continue
        done_ids.append(id)
        # try:
        s_time = time.time()
        info = getBookInfo(id)
        e_time = time.time()
        print('获取图书信息用时: %s 秒' % str(e_time - s_time))
        s_time = time.time()
        res = saveBookInfo(info)
        e_time = time.time()
        print('保存图书信息用时: %s 秒' % str(e_time - s_time))
        if res['code'] == 201 or res['code'] == 200:
            success_id(id)
            show_info(id, True)
        else:
            error_id(id, res['msg'])
            show_info(id, False)
        # except BaseException as e:
        #     error_id(id, e)
        #     done_ids.append(id)
        #     show_info(id, False)

        # time.sleep(random.randint(6,10))
    print('%s : Program stop...' % get_time())